Merge branch 'main' into playwight-ui-setup

.github/workflows/git-create-release-branch.yml

@@ -0,0 +1,33 @@
+name: Create Release Branches
+run-name: Create Release Branch ${{ inputs.release_branch_name }}
+on:
+  workflow_dispatch:
+    inputs:
+      release_branch_name:
+        description: "Github Release Branch Name"
+        required: true
+      base_branch_name:
+        description: "Base Branch for Release Branch"
+        required: true
+        default: "main"
+permissions:
+  contents: write
+jobs:
+  create-release-branch:
+    name: Create Release Branch ${{ inputs.release_branch_name }}
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Repo
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.base_branch_name }}
+    - name: Update application versions
+      run: |
+        make update_all RELEASE_VERSION=${{ inputs.release_branch_name }}
+    - name: Commit changes to ${{ inputs.release_branch_name }} branch
+      uses: EndBug/add-and-commit@v9
+      with:
+        default_author: github_actions
+        message: 'chore(release): Prepare Branch for `${{ inputs.release_branch_name }}`'
+        add: '.'
+        new_branch: ${{ inputs.release_branch_name }}

bootstrap/sql/migrations/native/1.4.2/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,5 @@
+-- Fix hanging partition migration
+-- in 1.4.x `tablePartition.intervalType` should not exists
+UPDATE table_entity
+SET json = JSON_REMOVE(json, '$.tablePartition')
+WHERE JSON_EXTRACT(json, '$.tablePartition.intervalType') IS NOT NULL;

bootstrap/sql/migrations/native/1.4.2/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,5 @@
+-- Fix hanging partition migration
+-- in 1.4.x `tablePartition.intervalType` should not exists
+UPDATE table_entity
+SET json = json - 'tablePartition'
+WHERE json->'tablePartition'->'intervalType' is not null;

ingestion/src/metadata/examples/workflows/kafka.yaml

@@ -14,6 +14,19 @@ source:
         # example of scema registry config, for more info about accepted values visit:
         # https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient
         #basic.auth.user.info: username:password
+      # schemaRegistrySSL:
+      #   caCertificate: |
+      #     -----BEGIN CERTIFICATE-----
+      #     sample caCertificateData
+      #     -----END CERTIFICATE-----
+      #   sslCertificate: |
+      #     -----BEGIN CERTIFICATE-----
+      #     sample sslCertificateData
+      #     -----END CERTIFICATE-----
+      #   sslKey: |
+      #     -----BEGIN RSA PRIVATE KEY
+      #     sample sslKeyData
+      #     -----END RSA PRIVATE KEY
   sourceConfig:
     config:
       type: MessagingMetadata

ingestion/src/metadata/examples/workflows/kafka_ssl_config.yaml

@@ -0,0 +1,46 @@
+source:
+  type: kafka
+  serviceName: local_kafka
+  serviceConnection:
+    config:
+      type: Kafka
+      bootstrapServers: localhost:9092
+      schemaRegistryURL: http://localhost:8081
+      #consumerConfig:
+        # example of consume config, for more info about accepted values visit:
+        # https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md
+        #ssl.truststore.password: password
+      #schemaRegistryConfig:
+        # example of scema registry config, for more info about accepted values visit:
+        # https://docs.confluent.io/5.5.1/clients/confluent-kafka-python/index.html#confluent_kafka.schema_registry.SchemaRegistryClient
+        #basic.auth.user.info: username:password
+      schemaRegistrySSL:
+        caCertificate: |
+          -----BEGIN CERTIFICATE-----
+          sample caCertificateData
+          -----END CERTIFICATE-----
+        sslCertificate: |
+          -----BEGIN CERTIFICATE-----
+          sample sslCertificateData
+          -----END CERTIFICATE-----
+        sslKey: |
+          -----BEGIN RSA PRIVATE KEY
+          sample sslKeyData
+          -----END RSA PRIVATE KEY
+  sourceConfig:
+    config:
+      type: MessagingMetadata
+      topicFilterPattern:
+        excludes:
+        - _confluent.*
+      generateSampleData: true
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+#  loggerLevel: INFO # DEBUG, INFO, WARN or ERROR
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      jwtToken: "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

ingestion/src/metadata/ingestion/source/database/common_db_source.py

@@ -510,7 +510,7 @@ def yield_table(
             self.register_record(table_request=table_request)
 
             # Flag view as visited
-            if table_type == TableType.View or schema_definition:
+            if table_type == TableType.View and schema_definition:
                 table_view = TableView.parse_obj(
                     {
                         "table_name": table_name,

ingestion/src/metadata/ingestion/source/database/databricks/metadata.py

@@ -43,6 +43,7 @@
 from metadata.ingestion.source.database.column_type_parser import create_sqlalchemy_type
 from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
 from metadata.ingestion.source.database.databricks.queries import (
+    DATABRICKS_DDL,
     DATABRICKS_GET_CATALOGS,
     DATABRICKS_GET_CATALOGS_TAGS,
     DATABRICKS_GET_COLUMN_TAGS,
@@ -60,9 +61,7 @@
 from metadata.utils.filters import filter_by_database
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import (
-    get_all_table_ddls,
     get_all_view_definitions,
-    get_table_ddl,
     get_view_definition_wrapper,
 )
 from metadata.utils.tag_utils import get_ometa_tag_and_classification
@@ -246,14 +245,32 @@ def get_view_definition(
     return None
 
 
+@reflection.cache
+def get_table_ddl(
+    self, connection, table_name, schema=None, **kw
+):  # pylint: disable=unused-argument
+    """
+    Gets the Table DDL
+    """
+    schema = schema or self.default_schema_name
+    table_name = f"{schema}.{table_name}" if schema else table_name
+    cursor = connection.execute(DATABRICKS_DDL.format(table_name=table_name))
+    try:
+        result = cursor.fetchone()
+        if result:
+            return result[0]
+    except Exception:
+        pass
+    return None
+
+
 DatabricksDialect.get_table_comment = get_table_comment
 DatabricksDialect.get_view_names = get_view_names
 DatabricksDialect.get_columns = get_columns
 DatabricksDialect.get_schema_names = get_schema_names
 DatabricksDialect.get_view_definition = get_view_definition
 DatabricksDialect.get_all_view_definitions = get_all_view_definitions
 reflection.Inspector.get_schema_names = get_schema_names_reflection
-reflection.Inspector.get_all_table_ddls = get_all_table_ddls
 reflection.Inspector.get_table_ddl = get_table_ddl
 
 

ingestion/src/metadata/ingestion/source/database/databricks/queries.py

@@ -54,3 +54,5 @@
     FROM {database_name}.information_schema.column_tags 
     """
 )
+
+DATABRICKS_DDL = "SHOW CREATE TABLE {table_name}"

ingestion/src/metadata/ingestion/source/database/postgres/usage.py

@@ -39,9 +39,11 @@ def process_table_query(self) -> Iterable[TableQueries]:
         """
         Process Query
         """
+        query = None
         try:
+            query = self.get_sql_statement()
             with get_connection(self.service_connection).connect() as conn:
-                rows = conn.execute(self.get_sql_statement())
+                rows = conn.execute(query)
                 queries = []
                 for row in rows:
                     row = dict(row)
@@ -64,5 +66,9 @@ def process_table_query(self) -> Iterable[TableQueries]:
             if queries:
                 yield TableQueries(queries=queries)
         except Exception as err:
+            if query:
+                logger.debug(
+                    f"###### USAGE QUERY #######\n{query}\n##########################"
+                )
             logger.error(f"Source usage processing error - {err}")
             logger.debug(traceback.format_exc())

ingestion/src/metadata/ingestion/source/database/usage_source.py

@@ -93,15 +93,15 @@ def yield_table_queries(self) -> Iterable[TableQuery]:
                 f"Scanning query logs for {(self.start + timedelta(days=days)).date()} - "
                 f"{(self.start + timedelta(days=days + 1)).date()}"
             )
+            query = None
             try:
+                query = self.get_sql_statement(
+                    start_time=self.start + timedelta(days=days),
+                    end_time=self.start + timedelta(days=days + 1),
+                )
                 for engine in self.get_engine():
                     with engine.connect() as conn:
-                        rows = conn.execute(
-                            self.get_sql_statement(
-                                start_time=self.start + timedelta(days=days),
-                                end_time=self.start + timedelta(days=days + 1),
-                            )
-                        )
+                        rows = conn.execute(query)
                         queries = []
                         for row in rows:
                             row = dict(row)
@@ -133,6 +133,10 @@ def yield_table_queries(self) -> Iterable[TableQuery]:
                                 )
                     yield TableQueries(queries=queries)
             except Exception as exc:
+                if query:
+                    logger.debug(
+                        f"###### USAGE QUERY #######\n{query}\n##########################"
+                    )
                 logger.debug(traceback.format_exc())
                 logger.error(f"Source usage processing error: {exc}")
 

ingestion/src/metadata/ingestion/source/messaging/kafka/metadata.py

@@ -11,7 +11,7 @@
 """
 Kafka source ingestion
 """
-from typing import Optional
+from typing import Optional, cast
 
 from metadata.generated.schema.entity.services.connections.messaging.kafkaConnection import (
     KafkaConnection,
@@ -28,7 +28,9 @@
 class KafkaSource(CommonBrokerSource):
     def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
         self.ssl_manager = None
-        service_connection = config.serviceConnection.__root__.config
+        service_connection = cast(
+            KafkaConnection, config.serviceConnection.__root__.config
+        )
         if service_connection.schemaRegistrySSL:
 
             self.ssl_manager = SSLManager(
@@ -37,7 +39,7 @@ def __init__(self, config: WorkflowSource, metadata: OpenMetadata):
                 cert=service_connection.schemaRegistrySSL.__root__.sslCertificate,
             )
             service_connection = self.ssl_manager.setup_ssl(
-                config.serviceConnection.__root__.config.sslConfig
+                config.serviceConnection.__root__.config
             )
         super().__init__(config, metadata)
 

ingestion/tests/unit/test_ssl_manager.py

@@ -1,15 +1,28 @@
 """
 Manage SSL test cases
 """
+
 import os
-import unittest
+from unittest import TestCase
+from unittest.mock import patch
 
 from pydantic import SecretStr
 
+from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
+    OpenMetadataConnection,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    Source as WorkflowSource,
+)
+from metadata.generated.schema.security.client.openMetadataJWTClientConfig import (
+    OpenMetadataJWTClientConfig,
+)
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.messaging.kafka.metadata import KafkaSource
 from metadata.utils.ssl_manager import SSLManager
 
 
-class SSLManagerTest(unittest.TestCase):
+class SSLManagerTest(TestCase):
     """
     Tests to verify the functionality of SSLManager
     """
@@ -44,3 +57,70 @@ def test_cleanup_temp_files(self):
         temp_file = self.ssl_manager.create_temp_file(SecretStr("Test content"))
         self.ssl_manager.cleanup_temp_files()
         self.assertFalse(os.path.exists(temp_file))
+
+
+class KafkaSourceSSLTest(TestCase):
+    @patch(
+        "metadata.ingestion.source.messaging.messaging_service.MessagingServiceSource.test_connection"
+    )
+    @patch("metadata.ingestion.source.messaging.kafka.metadata.SSLManager")
+    def test_init(self, mock_ssl_manager, test_connection):
+        test_connection.return_value = True
+        config = WorkflowSource(
+            **{
+                "type": "kafka",
+                "serviceName": "local_kafka",
+                "serviceConnection": {
+                    "config": {
+                        "type": "Kafka",
+                        "bootstrapServers": "localhost:9092",
+                    }
+                },
+                "sourceConfig": {"config": {"type": "MessagingMetadata"}},
+            }
+        )
+        metadata = OpenMetadata(
+            OpenMetadataConnection(
+                hostPort="http://localhost:8585/api",
+                authProvider="openmetadata",
+                securityConfig=OpenMetadataJWTClientConfig(jwtToken="token"),
+            )
+        )
+        kafka_source = KafkaSource(config, metadata)
+
+        self.assertIsNone(kafka_source.ssl_manager)
+        mock_ssl_manager.assert_not_called()
+
+        config_with_ssl = WorkflowSource(
+            **{
+                "type": "kafka",
+                "serviceName": "local_kafka",
+                "serviceConnection": {
+                    "config": {
+                        "type": "Kafka",
+                        "bootstrapServers": "localhost:9092",
+                        "schemaRegistrySSL": {
+                            "caCertificate": "caCertificateData",
+                            "sslKey": "sslKeyData",
+                            "sslCertificate": "sslCertificateData",
+                        },
+                    },
+                },
+                "sourceConfig": {"config": {"type": "MessagingMetadata"}},
+            }
+        )
+        kafka_source_with_ssl = KafkaSource(config_with_ssl, metadata)
+
+        self.assertIsNotNone(kafka_source_with_ssl.ssl_manager)
+        self.assertEqual(
+            kafka_source_with_ssl.service_connection.schemaRegistrySSL.__root__.caCertificate.get_secret_value(),
+            "caCertificateData",
+        )
+        self.assertEqual(
+            kafka_source_with_ssl.service_connection.schemaRegistrySSL.__root__.sslKey.get_secret_value(),
+            "sslKeyData",
+        )
+        self.assertEqual(
+            kafka_source_with_ssl.service_connection.schemaRegistrySSL.__root__.sslCertificate.get_secret_value(),
+            "sslCertificateData",
+        )

openmetadata-docs/content/v1.4.x/deployment/docker/index.md

@@ -102,7 +102,7 @@ This docker compose file contains only the docker compose services for OpenMetad
 You can also run the below command to fetch the docker compose file directly from the terminal -
 
 ```bash
-wget https://github.com/open-metadata/OpenMetadata/releases/download/1.3.1-release/docker-compose-openmetadata.yml
+wget https://github.com/open-metadata/OpenMetadata/releases/download/1.4.1-release/docker-compose-openmetadata.yml
 ```
 
 ### 3. Update Environment Variables required for OpenMetadata Dependencies
@@ -191,7 +191,7 @@ You can validate that all containers are up by running with command `docker ps`.
 ```commandline
 ❯ docker ps
 CONTAINER ID   IMAGE                                                  COMMAND                  CREATED          STATUS                    PORTS                                                            NAMES
-470cc8149826   openmetadata/server:1.3.1                              "./openmetadata-star…"   45 seconds ago   Up 43 seconds             3306/tcp, 9200/tcp, 9300/tcp, 0.0.0.0:8585-8586->8585-8586/tcp   openmetadata_server
+470cc8149826   openmetadata/server:1.4.1                              "./openmetadata-star…"   45 seconds ago   Up 43 seconds             3306/tcp, 9200/tcp, 9300/tcp, 0.0.0.0:8585-8586->8585-8586/tcp   openmetadata_server
 ```
 
 In a few seconds, you should be able to access the OpenMetadata UI at [http://localhost:8585](http://localhost:8585)