Adding support for MetaFrame (#440)

.github/workflows/release.yaml

@@ -15,5 +15,5 @@ jobs:
       - name: Create Release
         uses: SneaksAndData/github-actions/[email protected]
         with:
-          major_v: 2
-          minor_v: 11
+          major_v: 3
+          minor_v: 0

README.md

@@ -10,12 +10,12 @@ This project aim at providing tools needed for everyday activities of data scien
 
 This module provides basic Delta Lake operations without Spark session, based on [delta-rs](https://github.com/delta-io/delta-rs) project.
 
-Please refer to the [module](adapta/storage/delta_lake/README.md) documentation for examples.
+Please refer to the [module](adapta/storage/delta_lake/v3/README.md) documentation for examples.
 
 ## Secret Storages
 
 Please refer to the [module](adapta/storage/secrets/README.md) documentation for examples.
 
 ## NoSql (Astra DB)
 
-Please refer to the [module](adapta/storage/distributed_object_store/datastax_astra/README.md) documentation for examples.
+Please refer to the [module](adapta/storage/distributed_object_store/v3/datastax_astra/README.md) documentation for examples.

adapta/storage/database/__init__.py

@@ -1,3 +1,6 @@
+"""
+Import index
+"""
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,3 +15,5 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+
+from adapta.storage.database.v2 import *

adapta/storage/database/README.md → adapta/storage/database/v2/README.md

@@ -1,5 +1,7 @@
 # Database Clients
 
+**This is a deprecated module. Please use the new module `adapta.storage.database.v3` instead.**
+
 Supported clients:
 
 - Generic ODBC

adapta/storage/database/v2/__init__.py

@@ -0,0 +1,21 @@
+"""
+Import index
+"""
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+from adapta.storage.database.v2.azure_sql import *
+from adapta.storage.database.v2.snowflake_sql import *
+from adapta.storage.database.v2.odbc import *
+from adapta.storage.database.v2.trino_sql import *

adapta/storage/database/v2/azure_sql.py

@@ -0,0 +1,144 @@
+# pylint: disable=duplicate-code
+"""
+ ODBC client extension for Azure SQL.
+"""
+
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from typing import Optional
+from warnings import warn
+
+from sqlalchemy import text
+
+from adapta.logs import SemanticLogger
+from adapta.storage.database.v2.odbc import OdbcClient
+from adapta.storage.database.v2.models import DatabaseType
+from adapta.utils import doze
+
+
+class AzureSqlClient(OdbcClient):
+    """
+    Azure SQL (cloud) ODBC client.
+    """
+
+    def __init__(
+        self,
+        logger: SemanticLogger,
+        host_name: str,
+        user_name: str,
+        password: str,
+        database: Optional[str] = None,
+        port: Optional[int] = 1433,
+        database_type: Optional[DatabaseType] = DatabaseType.SQL_SERVER_ODBC,
+    ):
+        """
+          Creates an instance of an Azure SQL ODBC client.
+
+        :param logger: Logger instance for database operations.
+        :param host_name: Hostname of the instance.
+        :param user_name: User to connect with
+        :param password: SQL user password to use with this instance.
+        :param database: Database to connect to.
+        :param port: Connection port. Defaults to 1433.
+        """
+        super().__init__(
+            logger=logger,
+            database_type=database_type,
+            host_name=host_name,
+            user_name=user_name,
+            database=database,
+            password=password,
+            port=port,
+        )
+        warn(
+            "You are using version 2 of the AzureSqlClient class. "
+            "This is deprecated and will be removed in adapta version 4. "
+            "Please upgrade to version 3: adapta.storage.database.v3",
+            DeprecationWarning,
+        )
+
+    @property
+    def size(self) -> str:
+        """
+        Current size (Service Objective) of a database in Azure.
+        """
+        return get_current_objective(self)
+
+    def scale_instance(self, target_objective="HS_Gen4_8", max_wait_time: Optional[int] = 180) -> bool:
+        """
+          Scales up/down the connected database.
+
+        :param target_objective: Target Azure SQL instance size.
+        :param max_wait_time: If provided, waits for the operation to complete within a specified interval in seconds.
+          If a scale operation doesn't complete within the specified period, function will return False, otherwise True.
+        :return: Result of a scale operation.
+          NB: if a timeout is not specified, True is returned,
+          thus a user should perform a self-check if a downstream operation requires a scaled database.
+        """
+
+        assert self._database, "Database name must be provided when constructing a client for this method to execute."
+
+        current_objective = get_current_objective(self)
+
+        if current_objective == target_objective:
+            return True
+
+        _ = self._get_connection().execute(
+            text(f"ALTER DATABASE [{self._database}] MODIFY (service_objective = '{target_objective}');")
+        )
+
+        self._logger.info(
+            "Requested scale-up for {host}/{database}",
+            host=self._host,
+            database=self._database,
+        )
+
+        if max_wait_time:
+            elapsed = 0
+            while current_objective != target_objective and max_wait_time > elapsed:
+                elapsed += doze(60) // 1e9
+                with self.fork() as client_fork:
+                    current_objective = get_current_objective(client_fork)
+                    self._logger.info(
+                        "Waiting for the scale-up to complete, elapsed {elapsed}s",
+                        elapsed=elapsed,
+                    )
+
+            self._logger.info(
+                "Scale-up {result} after {elapsed}s",
+                result="completed" if current_objective == target_objective else "failed",
+                elapsed=elapsed,
+            )
+
+            return current_objective == target_objective
+
+        self._logger.info("Timeout not specified - exiting without awaiting the operation result")
+
+        return True
+
+
+def get_current_objective(client: AzureSqlClient) -> str:
+    """
+     Reads current database size for the specified client.
+
+    :param client: Azure SQL database (ODBC) client.
+    :return: Name of an active Azure SQL Service Objective.
+    """
+    return (
+        client.query("SELECT service_objective FROM sys.database_service_objectives")
+        .to_dict()
+        .get("service_objective", None)
+    )

adapta/storage/distributed_object_store/datastax_astra/__init__.py → adapta/storage/database/v2/models/__init__.py

@@ -1,3 +1,6 @@
+"""
+ Import index.
+"""
 #  Copyright (c) 2023-2024. ECCO Sneaks & Data
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +16,4 @@
 #  limitations under the License.
 #
 
-"""
- Import index.
-"""
-from adapta.storage.distributed_object_store.datastax_astra._models import *
+from adapta.storage.database.v2.models._models import *

adapta/storage/database/v2/models/_models.py

@@ -0,0 +1,54 @@
+# pylint: disable=duplicate-code
+"""
+  Models for relational database clients.
+"""
+#  Copyright (c) 2023-2024. ECCO Sneaks & Data
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict
+
+
+@dataclass
+class SqlAlchemyDialect:
+    """
+    Configuration for SqlAlchemy Engine Dialect
+    """
+
+    dialect: str
+    driver: Dict[str, str]
+
+
+class DatabaseType(Enum):
+    """
+    Pre-configured SqlAlchemy dialects for various clients.
+    """
+
+    SQL_SERVER_ODBC = SqlAlchemyDialect(
+        dialect="mssql+pyodbc",
+        driver={
+            "driver": "ODBC Driver 17 for SQL Server",
+            "LongAsMax": "Yes",
+        },
+    )
+    SQL_SERVER_ODBC_V18 = SqlAlchemyDialect(
+        dialect="mssql+pyodbc",
+        driver={
+            "driver": "ODBC Driver 18 for SQL Server",
+            "LongAsMax": "Yes",
+        },
+    )
+    SQLITE_ODBC = SqlAlchemyDialect(dialect="sqlite+pysqlite", driver={})

adapta/storage/database/odbc.py → adapta/storage/database/v2/odbc.py

@@ -1,3 +1,4 @@
+# pylint: disable=duplicate-code
 """
  Database client that uses an ODBC driver.
 """
@@ -19,6 +20,7 @@
 
 from abc import ABC
 from typing import Optional, Union, Iterator
+from warnings import warn
 
 from pandas import DataFrame, read_sql
 import sqlalchemy
@@ -28,7 +30,7 @@
 from sqlalchemy.exc import SQLAlchemyError, OperationalError
 
 from adapta.logs import SemanticLogger
-from adapta.storage.database.models import DatabaseType, SqlAlchemyDialect
+from adapta.storage.database.v2.models import DatabaseType, SqlAlchemyDialect
 
 
 class OdbcClient(ABC):
@@ -57,6 +59,12 @@ def __init__(
         :param password: SQL user password.
         :param port: Connection port.
         """
+        warn(
+            "You are using version 2 of the OdbcClient class. "
+            "This is deprecated and will be removed in adapta version 4. "
+            "Please upgrade to version 3: adapta.storage.database.v3",
+            DeprecationWarning,
+        )
         self._db_type = database_type
         self._dialect: SqlAlchemyDialect = database_type.value
         self._host = host_name

adapta/storage/database/snowflake_sql.py → adapta/storage/database/v2/snowflake_sql.py

@@ -1,3 +1,4 @@
+# pylint: disable=duplicate-code
 """
   Snowflake Client Wrapper
 """
@@ -6,6 +7,7 @@
 import re
 from types import TracebackType
 from typing import List, Optional, Dict
+from warnings import warn
 
 from pandas import DataFrame
 import snowflake.connector
@@ -46,6 +48,12 @@ def __init__(
         password: Optional[str] = None,
         role: Optional[str] = None,
     ):
+        warn(
+            "You are using version 2 of the SnowflakeClient class. "
+            "This is deprecated and will be removed in adapta version 4. "
+            "Please upgrade to version 3: adapta.storage.database.v3",
+            DeprecationWarning,
+        )
         self._user = user
         self._account = account
         self._warehouse = warehouse