diff --git a/poetry.lock b/poetry.lock index 397b2d01f..a51d96543 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3373,29 +3373,29 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.7.4" +version = "0.8.0" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.7.4-py3-none-linux_armv6l.whl", hash = "sha256:a4919925e7684a3f18e18243cd6bea7cfb8e968a6eaa8437971f681b7ec51478"}, - {file = "ruff-0.7.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:cfb365c135b830778dda8c04fb7d4280ed0b984e1aec27f574445231e20d6c63"}, - {file = "ruff-0.7.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:63a569b36bc66fbadec5beaa539dd81e0527cb258b94e29e0531ce41bacc1f20"}, - {file = "ruff-0.7.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d06218747d361d06fd2fdac734e7fa92df36df93035db3dc2ad7aa9852cb109"}, - {file = "ruff-0.7.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e0cea28d0944f74ebc33e9f934238f15c758841f9f5edd180b5315c203293452"}, - {file = "ruff-0.7.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80094ecd4793c68b2571b128f91754d60f692d64bc0d7272ec9197fdd09bf9ea"}, - {file = "ruff-0.7.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:997512325c6620d1c4c2b15db49ef59543ef9cd0f4aa8065ec2ae5103cedc7e7"}, - {file = "ruff-0.7.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:00b4cf3a6b5fad6d1a66e7574d78956bbd09abfd6c8a997798f01f5da3d46a05"}, - {file = "ruff-0.7.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7dbdc7d8274e1422722933d1edddfdc65b4336abf0b16dfcb9dedd6e6a517d06"}, - {file = "ruff-0.7.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e92dfb5f00eaedb1501b2f906ccabfd67b2355bdf117fea9719fc99ac2145bc"}, - {file = "ruff-0.7.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3bd726099f277d735dc38900b6a8d6cf070f80828877941983a57bca1cd92172"}, - {file = "ruff-0.7.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2e32829c429dd081ee5ba39aef436603e5b22335c3d3fff013cd585806a6486a"}, - {file = "ruff-0.7.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:662a63b4971807623f6f90c1fb664613f67cc182dc4d991471c23c541fee62dd"}, - {file = "ruff-0.7.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:876f5e09eaae3eb76814c1d3b68879891d6fde4824c015d48e7a7da4cf066a3a"}, - {file = "ruff-0.7.4-py3-none-win32.whl", hash = "sha256:75c53f54904be42dd52a548728a5b572344b50d9b2873d13a3f8c5e3b91f5cac"}, - {file = "ruff-0.7.4-py3-none-win_amd64.whl", hash = "sha256:745775c7b39f914238ed1f1b0bebed0b9155a17cd8bc0b08d3c87e4703b990d6"}, - {file = "ruff-0.7.4-py3-none-win_arm64.whl", hash = "sha256:11bff065102c3ae9d3ea4dc9ecdfe5a5171349cdd0787c1fc64761212fc9cf1f"}, - {file = "ruff-0.7.4.tar.gz", hash = "sha256:cd12e35031f5af6b9b93715d8c4f40360070b2041f81273d0527683d5708fce2"}, + {file = "ruff-0.8.0-py3-none-linux_armv6l.whl", hash = "sha256:fcb1bf2cc6706adae9d79c8d86478677e3bbd4ced796ccad106fd4776d395fea"}, + {file = "ruff-0.8.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:295bb4c02d58ff2ef4378a1870c20af30723013f441c9d1637a008baaf928c8b"}, + {file = "ruff-0.8.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7b1f1c76b47c18fa92ee78b60d2d20d7e866c55ee603e7d19c1e991fad933a9a"}, + {file = "ruff-0.8.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb0d4f250a7711b67ad513fde67e8870109e5ce590a801c3722580fe98c33a99"}, + {file = "ruff-0.8.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e55cce9aa93c5d0d4e3937e47b169035c7e91c8655b0974e61bb79cf398d49c"}, + {file = "ruff-0.8.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f4cd64916d8e732ce6b87f3f5296a8942d285bbbc161acee7fe561134af64f9"}, + {file = "ruff-0.8.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:c5c1466be2a2ebdf7c5450dd5d980cc87c8ba6976fb82582fea18823da6fa362"}, + {file = "ruff-0.8.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2dabfd05b96b7b8f2da00d53c514eea842bff83e41e1cceb08ae1966254a51df"}, + {file = "ruff-0.8.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:facebdfe5a5af6b1588a1d26d170635ead6892d0e314477e80256ef4a8470cf3"}, + {file = "ruff-0.8.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87a8e86bae0dbd749c815211ca11e3a7bd559b9710746c559ed63106d382bd9c"}, + {file = "ruff-0.8.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:85e654f0ded7befe2d61eeaf3d3b1e4ef3894469cd664ffa85006c7720f1e4a2"}, + {file = "ruff-0.8.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:83a55679c4cb449fa527b8497cadf54f076603cc36779b2170b24f704171ce70"}, + {file = "ruff-0.8.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:812e2052121634cf13cd6fddf0c1871d0ead1aad40a1a258753c04c18bb71bbd"}, + {file = "ruff-0.8.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:780d5d8523c04202184405e60c98d7595bdb498c3c6abba3b6d4cdf2ca2af426"}, + {file = "ruff-0.8.0-py3-none-win32.whl", hash = "sha256:5fdb6efecc3eb60bba5819679466471fd7d13c53487df7248d6e27146e985468"}, + {file = "ruff-0.8.0-py3-none-win_amd64.whl", hash = "sha256:582891c57b96228d146725975fbb942e1f30a0c4ba19722e692ca3eb25cc9b4f"}, + {file = "ruff-0.8.0-py3-none-win_arm64.whl", hash = "sha256:ba93e6294e9a737cd726b74b09a6972e36bb511f9a102f1d9a7e1ce94dd206a6"}, + {file = "ruff-0.8.0.tar.gz", hash = "sha256:a7ccfe6331bf8c8dad715753e157457faf7351c2b69f62f32c165c2dbcbacd44"}, ] [[package]] @@ -4313,4 +4313,4 @@ soap = ["lxml", "pandas", "pandas", "zeep"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.13" -content-hash = "08e2416fb0d4aa79f2b234196d4c5621ebbe3dd52c05e1ff3b13265737caee90" +content-hash = "fc0fdc545e8dd0810f7915f765ad4c5a982e8cafb385567480c408e78a19fd3d" diff --git a/pyproject.toml b/pyproject.toml index 1cf7fb223..8bb3984c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -142,7 +142,7 @@ all = [ ] [tool.poetry.group.dev.dependencies] -ruff = ">=0.6.1,<0.8.0" +ruff = ">=0.6.1,<0.9.0" pre-commit = ">=3.6,<5.0" types-oauthlib = "^3.2.0.20240217" diff --git a/tests/awsathena/test_awsathena.py b/tests/awsathena/test_awsathena.py index 98a369e18..f73570782 100644 --- a/tests/awsathena/test_awsathena.py +++ b/tests/awsathena/test_awsathena.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional +from typing import Optional from unittest.mock import MagicMock import pandas as pd @@ -48,7 +48,7 @@ def mocked_boto_session(mocker): @pytest.fixture -def status_checks() -> List[str]: +def status_checks() -> list[str]: return [ "Host resolved", "Port opened", diff --git a/tests/google_big_query/test_google_big_query.py b/tests/google_big_query/test_google_big_query.py index 7feab8d90..7c2997eb0 100644 --- a/tests/google_big_query/test_google_big_query.py +++ b/tests/google_big_query/test_google_big_query.py @@ -1,6 +1,7 @@ import json +from collections.abc import Generator from os import environ -from typing import Any, Generator +from typing import Any from unittest.mock import patch import numpy as np diff --git a/tests/s3/test_s3.py b/tests/s3/test_s3.py index 30738f951..65d060a07 100644 --- a/tests/s3/test_s3.py +++ b/tests/s3/test_s3.py @@ -1,6 +1,7 @@ import tempfile +from collections.abc import Generator from datetime import datetime, timedelta -from typing import Any, Generator +from typing import Any import openpyxl import pandas as pd diff --git a/tests/test_connector.py b/tests/test_connector.py index 44ce07d07..17b69bc2b 100644 --- a/tests/test_connector.py +++ b/tests/test_connector.py @@ -1,5 +1,4 @@ from time import time -from typing import List import pandas as pd import pytest @@ -346,7 +345,7 @@ def get_model( schema_name: str | None = None, table_name: str | None = None, exclude_columns: bool = False, - ) -> List[TableInfo]: + ) -> list[TableInfo]: model = [("database", "schema", "type", "name", [{"name": "column", "type": "type"}])] return DiscoverableConnector.format_db_model(model) diff --git a/tests/test_query_manager.py b/tests/test_query_manager.py index fd6f4e14f..31ddc4f1d 100644 --- a/tests/test_query_manager.py +++ b/tests/test_query_manager.py @@ -1,11 +1,11 @@ -from typing import Dict, Optional +from typing import Optional import pytest from toucan_connectors.query_manager import QueryManager -def fixture_execute_method(execute_method, query: str, query_parameters: Optional[Dict]): +def fixture_execute_method(execute_method, query: str, query_parameters: Optional[dict]): return True diff --git a/tests/utils/test_datetime.py b/tests/utils/test_datetime.py index 040f06a39..5d2971297 100644 --- a/tests/utils/test_datetime.py +++ b/tests/utils/test_datetime.py @@ -1,10 +1,10 @@ from datetime import date, datetime, timezone +from zoneinfo import ZoneInfo import pandas as pd import pytest from dateutil import tz from numpy import dtype -from zoneinfo import ZoneInfo from toucan_connectors.utils.datetime import is_datetime_col, sanitize_df_dates diff --git a/toucan_connectors/awsathena/awsathena_connector.py b/toucan_connectors/awsathena/awsathena_connector.py index 106241e0a..5ce829ca0 100644 --- a/toucan_connectors/awsathena/awsathena_connector.py +++ b/toucan_connectors/awsathena/awsathena_connector.py @@ -1,9 +1,8 @@ import logging -from typing import Any +from typing import Annotated, Any from cached_property import cached_property_with_ttl from pydantic import ConfigDict, Field, StringConstraints, create_model -from typing_extensions import Annotated try: import awswrangler as wr diff --git a/toucan_connectors/azure_mssql/azure_mssql_connector.py b/toucan_connectors/azure_mssql/azure_mssql_connector.py index f13d7d379..c3ddcbf23 100644 --- a/toucan_connectors/azure_mssql/azure_mssql_connector.py +++ b/toucan_connectors/azure_mssql/azure_mssql_connector.py @@ -10,8 +10,9 @@ getLogger(__name__).warning(f"Missing dependencies for {__name__}: {exc}") CONNECTOR_OK = False +from typing import Annotated + from pydantic import Field, StringConstraints -from typing_extensions import Annotated from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import PlainJsonSecretStr, ToucanConnector, ToucanDataSource diff --git a/toucan_connectors/clickhouse/clickhouse_connector.py b/toucan_connectors/clickhouse/clickhouse_connector.py index c21db57ce..d85341fba 100644 --- a/toucan_connectors/clickhouse/clickhouse_connector.py +++ b/toucan_connectors/clickhouse/clickhouse_connector.py @@ -10,9 +10,10 @@ getLogger(__name__).warning(f"Missing dependencies for {__name__}: {exc}") CONNECTOR_OK = False +from typing import Annotated + from pydantic import Field, StringConstraints, create_model from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode -from typing_extensions import Annotated from toucan_connectors.common import pandas_read_sql from toucan_connectors.toucan_connector import ( diff --git a/toucan_connectors/condition_translator.py b/toucan_connectors/condition_translator.py index ad5a9285c..72c264513 100644 --- a/toucan_connectors/condition_translator.py +++ b/toucan_connectors/condition_translator.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from enum import Enum -from typing import Any, List, Literal, TypeVar, Union +from typing import Any, Literal, TypeVar, Union from pydantic import BaseModel @@ -86,7 +86,7 @@ def translate(cls, condition: dict): @classmethod @abstractmethod - def join_clauses(cls, clauses: List[ClauseType], logical_operator: LogicalOperator): + def join_clauses(cls, clauses: list[ClauseType], logical_operator: LogicalOperator): """ Join multiple clauses with `and` or `or`. """ @@ -142,13 +142,13 @@ def GREATER_THAN_EQUAL(cls, column: str, value: Number) -> ClauseType: @classmethod @abstractmethod - def IN(cls, column: str, values: List[str]) -> ClauseType: + def IN(cls, column: str, values: list[str]) -> ClauseType: """`column` values in `values`""" raise NotImplementedError @classmethod @abstractmethod - def NOT_IN(cls, column: str, values: List[str]) -> ClauseType: + def NOT_IN(cls, column: str, values: list[str]) -> ClauseType: """`column` values not in `values`""" raise NotImplementedError diff --git a/toucan_connectors/connection_manager.py b/toucan_connectors/connection_manager.py index 15812efdc..51114c296 100644 --- a/toucan_connectors/connection_manager.py +++ b/toucan_connectors/connection_manager.py @@ -4,7 +4,7 @@ import time import types from enum import Enum -from typing import Dict, Optional, Union +from typing import Optional, Union logger = logging.getLogger(__name__) @@ -99,7 +99,7 @@ def force_to_remove(self): class ConnectionManager: def __init__(self, **kwargs): self.name: str = "connection_manager" - self.connection_list: Dict[str, ConnectionBO] = {} + self.connection_list: dict[str, ConnectionBO] = {} self.timeout = 5 self.wait = 0.2 diff --git a/toucan_connectors/databricks/databricks_connector.py b/toucan_connectors/databricks/databricks_connector.py index 8dc474a96..f5d4bbca6 100644 --- a/toucan_connectors/databricks/databricks_connector.py +++ b/toucan_connectors/databricks/databricks_connector.py @@ -1,8 +1,7 @@ import logging -from typing import List, Optional, Tuple +from typing import Annotated, Optional from pydantic import Field, StringConstraints -from typing_extensions import Annotated from toucan_connectors.common import ClusterStartException, ConnectorStatus, pandas_read_sql from toucan_connectors.toucan_connector import PlainJsonSecretStr, ToucanConnector, ToucanDataSource @@ -69,7 +68,7 @@ def _build_connection_string(self) -> str: return ";".join(f"{k}={v}" for k, v in connection_params.items() if v is not None) @staticmethod - def _get_details(index: int, status: Optional[bool]) -> List[Tuple[str, bool]]: + def _get_details(index: int, status: Optional[bool]) -> list[tuple[str, bool]]: checks = ["Host resolved", "Port opened", "Connected to Databricks", "Authenticated"] ok_checks = [(c, True) for i, c in enumerate(checks) if i < index] new_check = (checks[index], status) diff --git a/toucan_connectors/elasticsearch/elasticsearch_connector.py b/toucan_connectors/elasticsearch/elasticsearch_connector.py index 9a4b70164..2138861f3 100644 --- a/toucan_connectors/elasticsearch/elasticsearch_connector.py +++ b/toucan_connectors/elasticsearch/elasticsearch_connector.py @@ -1,7 +1,6 @@ from copy import deepcopy from enum import Enum from logging import getLogger -from typing import List from urllib.parse import urlparse try: @@ -125,7 +124,7 @@ class ElasticsearchDataSource(ToucanDataSource): class ElasticsearchConnector(ToucanConnector, data_source_model=ElasticsearchDataSource): - hosts: List[ElasticsearchHost] + hosts: list[ElasticsearchHost] def _retrieve_data(self, data_source: ElasticsearchDataSource) -> "pd.DataFrame": data_source.body = nosql_apply_parameters_to_query(data_source.body, data_source.parameters) diff --git a/toucan_connectors/github/github_connector.py b/toucan_connectors/github/github_connector.py index 70c789fc9..88ab51c74 100644 --- a/toucan_connectors/github/github_connector.py +++ b/toucan_connectors/github/github_connector.py @@ -5,7 +5,7 @@ from datetime import datetime from enum import Enum from pathlib import Path -from typing import List, Optional +from typing import Optional from dateutil import relativedelta from pydantic import Field, PrivateAttr, create_model @@ -209,7 +209,7 @@ async def get_pages( retries=0, retry_limit=2, latest_retrieved_object=None, - ) -> List[dict]: + ) -> list[dict]: """ Extracts pages of either members or pull requests :param name a str representing the repo name diff --git a/toucan_connectors/github/helpers.py b/toucan_connectors/github/helpers.py index 69a8467bb..b56b2285c 100644 --- a/toucan_connectors/github/helpers.py +++ b/toucan_connectors/github/helpers.py @@ -1,6 +1,5 @@ import logging from datetime import datetime -from typing import List import pandas as pd @@ -199,7 +198,7 @@ def format_pr_row(pr_row: dict): return current_record -def format_pr_rows(pr_nodes: dict, repo_name: str) -> List[dict]: +def format_pr_rows(pr_nodes: dict, repo_name: str) -> list[dict]: """ A wrapper function to multiprocess the pull requests formatting :param pr_rows: pull requests extracted from Github's API @@ -229,7 +228,7 @@ def format_team_row(members: dict, team_name: str) -> dict: return devs.to_dict().get("variable") -def format_team_df(team_rows: List[dict]) -> pd.DataFrame: +def format_team_df(team_rows: list[dict]) -> pd.DataFrame: """ Builds a Pandas DataFrame from members rows. :param team_rows: a list of dict with login as key and list @@ -312,7 +311,7 @@ def get_teams(organization: dict): raise KeyNotFoundException("No teams Key Available") -def get_nodes(response: dict) -> List[dict]: +def get_nodes(response: dict) -> list[dict]: """ Extracts value from a dict with nodes key or raises an error if the key is not available :param response: a response from Github's API @@ -322,7 +321,7 @@ def get_nodes(response: dict) -> List[dict]: return nodes -def get_edges(data: dict) -> List[dict]: +def get_edges(data: dict) -> list[dict]: """ Extracts value from a dict with edges key or raises an error if the key is not available :param data: data extracted from Github's API diff --git a/toucan_connectors/google_big_query/google_big_query_connector.py b/toucan_connectors/google_big_query/google_big_query_connector.py index 9b5d1ed7b..7551996b8 100644 --- a/toucan_connectors/google_big_query/google_big_query_connector.py +++ b/toucan_connectors/google_big_query/google_big_query_connector.py @@ -1,11 +1,12 @@ import logging import re +from collections.abc import Iterable from contextlib import suppress from enum import Enum from functools import cached_property from itertools import groupby from timeit import default_timer as timer -from typing import Any, Iterable, Union +from typing import Any, Union from pydantic import ConfigDict, Field, create_model diff --git a/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py b/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py index 25e1ba667..d157eb26d 100644 --- a/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py +++ b/toucan_connectors/google_cloud_mysql/google_cloud_mysql_connector.py @@ -1,7 +1,7 @@ from logging import getLogger +from typing import Annotated from pydantic import Field, StringConstraints -from typing_extensions import Annotated try: import pandas as pd diff --git a/toucan_connectors/google_sheets/google_sheets_connector.py b/toucan_connectors/google_sheets/google_sheets_connector.py index 1206db35a..a1d4311ae 100644 --- a/toucan_connectors/google_sheets/google_sheets_connector.py +++ b/toucan_connectors/google_sheets/google_sheets_connector.py @@ -1,7 +1,7 @@ from contextlib import suppress from datetime import datetime from logging import getLogger -from typing import Any, Callable, List, Optional +from typing import Any, Callable, Optional try: import numpy as np @@ -110,7 +110,7 @@ def build_sheets_api(self): def build_oauth2(self): return build("oauth2", "v2", cache_discovery=False, **self._google_client_build_kwargs()) - def list_sheets(self, spreadsheet_id: str) -> List[str]: + def list_sheets(self, spreadsheet_id: str) -> list[str]: """ List available sheets """ diff --git a/toucan_connectors/http_api/http_api_connector.py b/toucan_connectors/http_api/http_api_connector.py index ab57e5b0b..a8c7ced1c 100644 --- a/toucan_connectors/http_api/http_api_connector.py +++ b/toucan_connectors/http_api/http_api_connector.py @@ -1,7 +1,7 @@ import json from enum import Enum from logging import getLogger -from typing import Any, List +from typing import Any from pydantic import AnyHttpUrl, BaseModel, Field, FilePath @@ -78,7 +78,7 @@ class Template(BaseModel): class HttpAPIConnector(ToucanConnector, data_source_model=HttpAPIDataSource): responsetype: ResponseType = Field(ResponseType.json, title="Content-type of response") baseroute: AnyHttpUrl = Field(..., title="Baseroute URL", description="Baseroute URL") - cert: List[FilePath] | None = Field(None, title="Certificate", description="File path of your certificate if any") + cert: list[FilePath] | None = Field(None, title="Certificate", description="File path of your certificate if any") auth: Auth | None = Field(None, title="Authentication type") template: Template | None = Field( None, diff --git a/toucan_connectors/hubspot_private_app/hubspot_connector.py b/toucan_connectors/hubspot_private_app/hubspot_connector.py index 8aa336ae3..2cecde3d0 100644 --- a/toucan_connectors/hubspot_private_app/hubspot_connector.py +++ b/toucan_connectors/hubspot_private_app/hubspot_connector.py @@ -1,7 +1,8 @@ +from collections.abc import Generator from contextlib import suppress from datetime import datetime from logging import getLogger -from typing import Any, Generator, Protocol, TypeAlias +from typing import Any, Protocol, TypeAlias from pydantic import BaseModel, ConfigDict, Field, create_model diff --git a/toucan_connectors/json_wrapper.py b/toucan_connectors/json_wrapper.py index f1a713f1e..71c598e60 100644 --- a/toucan_connectors/json_wrapper.py +++ b/toucan_connectors/json_wrapper.py @@ -1,6 +1,5 @@ import json import logging -from typing import Dict from pydantic import SecretStr @@ -88,7 +87,7 @@ def load( parse_constant=None, object_pairs_hook=None, **kwargs, - ) -> Dict: + ) -> dict: return JsonWrapper.loads( fp.read(), cls=cls, @@ -111,7 +110,7 @@ def loads( parse_constant=None, object_pairs_hook=None, **kwargs, - ) -> Dict: + ) -> dict: logging.getLogger(__name__).debug(f"Stringify JSON {s}") result = json.loads( s, diff --git a/toucan_connectors/mongo/mongo_connector.py b/toucan_connectors/mongo/mongo_connector.py index 4dd09a05b..e2d711f32 100644 --- a/toucan_connectors/mongo/mongo_connector.py +++ b/toucan_connectors/mongo/mongo_connector.py @@ -3,7 +3,8 @@ from contextlib import contextmanager from functools import _lru_cache_wrapper, cached_property, lru_cache from logging import getLogger -from typing import Any, List, Optional, Pattern, Union +from re import Pattern +from typing import Any, Optional, Union from warnings import warn try: @@ -337,7 +338,7 @@ def get_slice( def get_slice_with_regex( self, data_source: MongoDataSource, - search: dict[str, List[dict[str, Pattern]]], + search: dict[str, list[dict[str, Pattern]]], permissions: Optional[str] = None, limit: Optional[int] = None, offset: Optional[int] = None, @@ -372,7 +373,7 @@ def get_slice_with_regex( def get_df_with_regex( self, data_source: MongoDataSource, - search: dict[str, List[dict[str, Pattern]]], + search: dict[str, list[dict[str, Pattern]]], permissions: Optional[str] = None, limit: Optional[int] = None, offset: Optional[int] = None, diff --git a/toucan_connectors/mongo/mongo_translator.py b/toucan_connectors/mongo/mongo_translator.py index 2f0b9cfcb..7a3083968 100644 --- a/toucan_connectors/mongo/mongo_translator.py +++ b/toucan_connectors/mongo/mongo_translator.py @@ -1,5 +1,3 @@ -from typing import Dict, List - from toucan_connectors.condition_translator import ConditionTranslator @@ -9,53 +7,53 @@ class MongoConditionTranslator(ConditionTranslator): """ @classmethod - def join_clauses(cls, clauses: List[dict], logical_operator: str): + def join_clauses(cls, clauses: list[dict], logical_operator: str): return {f"${logical_operator}": clauses} @classmethod - def EQUAL(cls, column, value) -> Dict[str, dict]: + def EQUAL(cls, column, value) -> dict[str, dict]: return {column: {"$eq": value}} @classmethod - def NOT_EQUAL(cls, column, value) -> Dict[str, dict]: + def NOT_EQUAL(cls, column, value) -> dict[str, dict]: return {column: {"$ne": value}} @classmethod - def LOWER_THAN(cls, column, value) -> Dict[str, dict]: + def LOWER_THAN(cls, column, value) -> dict[str, dict]: return {column: {"$lt": value}} @classmethod - def LOWER_THAN_EQUAL(cls, column, value) -> Dict[str, dict]: + def LOWER_THAN_EQUAL(cls, column, value) -> dict[str, dict]: return {column: {"$lte": value}} @classmethod - def GREATER_THAN(cls, column, value) -> Dict[str, dict]: + def GREATER_THAN(cls, column, value) -> dict[str, dict]: return {column: {"$gt": value}} @classmethod - def GREATER_THAN_EQUAL(cls, column, value) -> Dict[str, dict]: + def GREATER_THAN_EQUAL(cls, column, value) -> dict[str, dict]: return {column: {"$gte": value}} @classmethod - def IN(cls, column, values) -> Dict[str, dict]: + def IN(cls, column, values) -> dict[str, dict]: return {column: {"$in": values}} @classmethod - def NOT_IN(cls, column, values) -> Dict[str, dict]: + def NOT_IN(cls, column, values) -> dict[str, dict]: return {column: {"$nin": values}} @classmethod - def MATCHES(cls, column, value) -> Dict[str, dict]: + def MATCHES(cls, column, value) -> dict[str, dict]: return {column: {"$regex": value}} @classmethod - def NOT_MATCHES(cls, column, value) -> Dict[str, dict]: + def NOT_MATCHES(cls, column, value) -> dict[str, dict]: return {column: {"$not": {"$regex": value}}} @classmethod - def IS_NULL(cls, column, value=None) -> Dict[str, dict]: + def IS_NULL(cls, column, value=None) -> dict[str, dict]: return {column: {"$exists": False}} @classmethod - def IS_NOT_NULL(cls, column, value=None) -> Dict[str, dict]: + def IS_NOT_NULL(cls, column, value=None) -> dict[str, dict]: return {column: {"$exists": True}} diff --git a/toucan_connectors/mssql/mssql_connector.py b/toucan_connectors/mssql/mssql_connector.py index 701aaba27..6f8305afc 100644 --- a/toucan_connectors/mssql/mssql_connector.py +++ b/toucan_connectors/mssql/mssql_connector.py @@ -1,8 +1,8 @@ from contextlib import suppress from logging import getLogger +from typing import Annotated from pydantic import Field, StringConstraints, create_model -from typing_extensions import Annotated try: import pyodbc diff --git a/toucan_connectors/mssql_TLSv1_0/mssql_connector.py b/toucan_connectors/mssql_TLSv1_0/mssql_connector.py index f394f58b8..85c9c8bc2 100644 --- a/toucan_connectors/mssql_TLSv1_0/mssql_connector.py +++ b/toucan_connectors/mssql_TLSv1_0/mssql_connector.py @@ -1,8 +1,8 @@ import logging from contextlib import suppress +from typing import Annotated from pydantic import Field, StringConstraints, create_model -from typing_extensions import Annotated try: import pyodbc diff --git a/toucan_connectors/mysql/mysql_connector.py b/toucan_connectors/mysql/mysql_connector.py index c22938351..d6e037585 100644 --- a/toucan_connectors/mysql/mysql_connector.py +++ b/toucan_connectors/mysql/mysql_connector.py @@ -1,14 +1,14 @@ import logging import os import re +from collections.abc import Generator from enum import Enum from itertools import groupby as groupby from tempfile import NamedTemporaryFile -from typing import Any, Generator, Optional +from typing import Annotated, Any, Optional from cached_property import cached_property_with_ttl from pydantic import ConfigDict, Field, StringConstraints, create_model, model_validator -from typing_extensions import Annotated from toucan_connectors.common import ( ConnectorStatus, diff --git a/toucan_connectors/odbc/odbc_connector.py b/toucan_connectors/odbc/odbc_connector.py index d55d6e725..1065c8603 100644 --- a/toucan_connectors/odbc/odbc_connector.py +++ b/toucan_connectors/odbc/odbc_connector.py @@ -1,7 +1,7 @@ from logging import getLogger +from typing import Annotated from pydantic import Field, StringConstraints -from typing_extensions import Annotated try: import pandas as pd diff --git a/toucan_connectors/one_drive/one_drive_connector.py b/toucan_connectors/one_drive/one_drive_connector.py index c76e06b4d..ab3ef6abe 100755 --- a/toucan_connectors/one_drive/one_drive_connector.py +++ b/toucan_connectors/one_drive/one_drive_connector.py @@ -1,6 +1,6 @@ import logging import re -from typing import List, Optional +from typing import Optional from pydantic import Field, PrivateAttr @@ -58,7 +58,7 @@ class OneDriveDataSource(ToucanDataSource): description="Read one table or append multiple tables", placeholder="Enter a table or a comma separated list of tables", ) - parse_dates: Optional[List[str]] = Field( + parse_dates: Optional[list[str]] = Field( [], Title="Date columns", description="By default, dates are converted in the number of days since 1900/01/01", @@ -224,7 +224,7 @@ def _run_fetch(self, url): return response.json() - def _retrieve_files_path(self, data_source: OneDriveDataSource) -> List[str]: + def _retrieve_files_path(self, data_source: OneDriveDataSource) -> list[str]: logging.getLogger(__name__).debug("_retrieve_files_path") path = None # Split the "file" input to retrieve the path & the pattern diff --git a/toucan_connectors/oracle_sql/oracle_sql_connector.py b/toucan_connectors/oracle_sql/oracle_sql_connector.py index 621476af7..acdb6731d 100644 --- a/toucan_connectors/oracle_sql/oracle_sql_connector.py +++ b/toucan_connectors/oracle_sql/oracle_sql_connector.py @@ -1,8 +1,8 @@ from contextlib import suppress from logging import getLogger +from typing import Annotated from pydantic import Field, StringConstraints, create_model -from typing_extensions import Annotated try: import cx_Oracle diff --git a/toucan_connectors/postgres/postgresql_connector.py b/toucan_connectors/postgres/postgresql_connector.py index 9aa496d12..fad5a68bd 100644 --- a/toucan_connectors/postgres/postgresql_connector.py +++ b/toucan_connectors/postgres/postgresql_connector.py @@ -1,8 +1,7 @@ from logging import getLogger -from typing import Optional +from typing import Annotated, Optional from pydantic import Field, StringConstraints, create_model -from typing_extensions import Annotated from toucan_connectors.common import ConnectorStatus, pandas_read_sql from toucan_connectors.postgres.utils import build_database_model_extraction_query, types diff --git a/toucan_connectors/query_manager.py b/toucan_connectors/query_manager.py index 51daf213c..2be528f47 100644 --- a/toucan_connectors/query_manager.py +++ b/toucan_connectors/query_manager.py @@ -1,6 +1,6 @@ import logging import types -from typing import Dict, Optional +from typing import Optional from toucan_connectors import DataSlice @@ -9,10 +9,10 @@ class QueryManager: def __init__(self): - self.query: Dict[str, DataSlice] = {} + self.query: dict[str, DataSlice] = {} @staticmethod - def _execute(execute_method, connection, query: str, parameters: Optional[Dict] = None): + def _execute(execute_method, connection, query: str, parameters: Optional[dict] = None): logger.debug("call execute method") if isinstance(execute_method, types.MethodType) or isinstance(execute_method, types.FunctionType): result = execute_method(connection, query, parameters) @@ -20,7 +20,7 @@ def _execute(execute_method, connection, query: str, parameters: Optional[Dict] else: raise TypeError("execute_method is not callable") - def execute(self, execute_method, connection, query: str, query_parameters: Optional[Dict] = None): + def execute(self, execute_method, connection, query: str, query_parameters: Optional[dict] = None): result = QueryManager._execute(execute_method, connection, query, query_parameters) return result diff --git a/toucan_connectors/redshift/redshift_database_connector.py b/toucan_connectors/redshift/redshift_database_connector.py index 929775a53..8459311fe 100644 --- a/toucan_connectors/redshift/redshift_database_connector.py +++ b/toucan_connectors/redshift/redshift_database_connector.py @@ -3,7 +3,7 @@ from contextlib import suppress from enum import Enum from functools import cached_property -from typing import Any, Type +from typing import Annotated, Any from pydantic import ( ConfigDict, @@ -14,7 +14,6 @@ model_validator, ) from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode -from typing_extensions import Annotated try: import pandas as pd @@ -160,7 +159,8 @@ def model_json_schema( cls, by_alias: bool = True, ref_template: str = DEFAULT_REF_TEMPLATE, - schema_generator: Type[GenerateJsonSchema] = GenerateJsonSchema, + # mypy thinkgs that `type` refers to ToucanConnector.type + schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, # type:ignore[valid-type] mode: JsonSchemaMode = "validation", ) -> dict[str, Any]: schema = super().model_json_schema( diff --git a/toucan_connectors/sap_hana/sap_hana_connector.py b/toucan_connectors/sap_hana/sap_hana_connector.py index a7ec0f95c..c2f09218b 100644 --- a/toucan_connectors/sap_hana/sap_hana_connector.py +++ b/toucan_connectors/sap_hana/sap_hana_connector.py @@ -1,7 +1,7 @@ from logging import getLogger +from typing import Annotated from pydantic import Field, StringConstraints -from typing_extensions import Annotated try: import pyhdb diff --git a/toucan_connectors/snowflake/snowflake_connector.py b/toucan_connectors/snowflake/snowflake_connector.py index 12bc73e5c..9416287e6 100644 --- a/toucan_connectors/snowflake/snowflake_connector.py +++ b/toucan_connectors/snowflake/snowflake_connector.py @@ -1,8 +1,9 @@ import logging -from contextlib import contextmanager, suppress +from collections.abc import Generator +from contextlib import AbstractContextManager, contextmanager, suppress from datetime import datetime from enum import Enum -from typing import Any, ContextManager, Generator, Literal, overload +from typing import Any, Literal, overload from pydantic import Field, create_model from pydantic.json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode @@ -335,7 +336,7 @@ def _refresh_oauth_token(self): def _get_connection( self, database: str | None = None, warehouse: str | None = None - ) -> ContextManager["SnowflakeConnection"]: + ) -> AbstractContextManager["SnowflakeConnection"]: if self.authentication_method == AuthenticationMethod.OAUTH: _LOGGER.info("Refreshing OAuth token...") self._refresh_oauth_token() diff --git a/toucan_connectors/snowflake_common.py b/toucan_connectors/snowflake_common.py index bc7443df4..4155b39fa 100644 --- a/toucan_connectors/snowflake_common.py +++ b/toucan_connectors/snowflake_common.py @@ -2,10 +2,9 @@ import json import logging from timeit import default_timer as timer -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Annotated, Any, Optional from pydantic import Field, StringConstraints -from typing_extensions import Annotated from toucan_connectors.pagination import build_pagination_info from toucan_connectors.query_manager import QueryManager @@ -50,7 +49,7 @@ class SfDataSource(ToucanDataSource): ..., description="You can write your SQL query here", widget="sql" ) - query_object: Dict = Field( + query_object: dict = Field( None, description="An object describing a simple select query" "For example " @@ -87,7 +86,7 @@ def __init__(self): self.data_conversion_time: Optional[float] = None self.data_filtered_from_permission_time: Optional[float] = None self.compute_stats_time: Optional[float] = None - self.column_names_and_types: Optional[Dict[str, str]] = None + self.column_names_and_types: Optional[dict[str, str]] = None def set_data(self, data): self.data = data.result() @@ -104,7 +103,7 @@ def set_query_generation_time(self, query_generation_time): def set_data_conversion_time(self, data_conversion_time): self.data_conversion_time = data_conversion_time - def _execute_query(self, connection, query: str, query_parameters: Optional[Dict] = None): + def _execute_query(self, connection, query: str, query_parameters: Optional[dict] = None): return QueryManager().execute( execute_method=self._execute_query_internal, connection=connection, @@ -172,7 +171,7 @@ def _execute_parallelized_queries( self, connection: "SnowflakeConnection", query: str, - query_parameters: Optional[Dict] = None, + query_parameters: Optional[dict] = None, offset: Optional[int] = None, limit: Optional[int] = None, get_row_count=False, @@ -288,14 +287,14 @@ def get_slice( pagination_info=result.pagination_info, ) - def get_warehouses(self, connection: "SnowflakeConnection", warehouse_name: Optional[str] = None) -> List[str]: + def get_warehouses(self, connection: "SnowflakeConnection", warehouse_name: Optional[str] = None) -> list[str]: query = "SHOW WAREHOUSES" if warehouse_name: query = f"{query} LIKE '{warehouse_name}'" res = self._execute_query(connection, query).to_dict().get("name") return list(res.values()) if res else [] - def get_databases(self, connection: "SnowflakeConnection", database_name: Optional[str] = None) -> List[str]: + def get_databases(self, connection: "SnowflakeConnection", database_name: Optional[str] = None) -> list[str]: query = "SHOW DATABASES" if database_name: query = f"{query} LIKE '{database_name}'" @@ -313,7 +312,7 @@ def _describe( self, connection: "SnowflakeConnection", query: str, - ) -> Dict[str, str]: + ) -> dict[str, str]: from snowflake.connector import DictCursor description_start = timer() @@ -336,6 +335,6 @@ def _describe( res = {r.name: type_code_mapping.get(r.type_code) for r in describe_res} return res - def get_db_content(self, connection: "SnowflakeConnection") -> List[Dict[str, Any]]: + def get_db_content(self, connection: "SnowflakeConnection") -> list[dict[str, Any]]: query = build_database_model_extraction_query() return self._execute_query(connection, query) diff --git a/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py b/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py index c9cdd66ab..85621cd08 100644 --- a/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py +++ b/toucan_connectors/snowflake_oauth2/snowflake_oauth2_connector.py @@ -3,7 +3,7 @@ import uuid from contextlib import suppress from timeit import default_timer as timer -from typing import Any, Dict, List, Optional +from typing import Any, Optional from pydantic import Field, PrivateAttr, create_model @@ -236,7 +236,7 @@ def get_identifier(self): def _get_unique_datasource_identifier(self, data_source: SnowflakeoAuth2DataSource) -> dict: return SnowflakeCommon().render_datasource(data_source) - def _get_warehouses(self, warehouse_name: Optional[str] = None) -> List[str]: + def _get_warehouses(self, warehouse_name: Optional[str] = None) -> list[str]: with self._get_connection(warehouse=warehouse_name) as connection: result = SnowflakeCommon().get_warehouses(connection, warehouse_name) return result @@ -247,7 +247,7 @@ def _set_warehouse(self, data_source: SnowflakeoAuth2DataSource): data_source.warehouse = self.default_warehouse return data_source - def _get_databases(self, database_name: Optional[str] = None) -> List[str]: + def _get_databases(self, database_name: Optional[str] = None) -> list[str]: with self._get_connection(database=database_name) as connection: result = SnowflakeCommon().get_databases(connection, database_name) return result @@ -275,7 +275,7 @@ def get_slice( ) return result - def describe(self, data_source: SnowflakeoAuth2DataSource) -> Dict[str, str]: + def describe(self, data_source: SnowflakeoAuth2DataSource) -> dict[str, str]: data_source = self._set_warehouse(data_source) with self._get_connection(data_source.database, data_source.warehouse) as connection: result = SnowflakeCommon().describe(connection, data_source.query) diff --git a/toucan_connectors/sql_query_helper.py b/toucan_connectors/sql_query_helper.py index 190fa85e3..9cddb46e2 100644 --- a/toucan_connectors/sql_query_helper.py +++ b/toucan_connectors/sql_query_helper.py @@ -1,5 +1,5 @@ import re -from typing import Dict, Optional, Tuple +from typing import Optional from toucan_connectors.common import convert_to_printf_templating_style, convert_to_qmark_paramstyle @@ -14,7 +14,7 @@ def count_query_needed( return bool(re.search(r"select.*", query, re.I)) @staticmethod - def prepare_count_query(query_string: str, query_parameters: Optional[Dict] = None) -> Tuple[str, list]: + def prepare_count_query(query_string: str, query_parameters: Optional[dict] = None) -> tuple[str, list]: """Build the count(*) query by removing the limit and the offset and adding a count query above from input query""" prepared_query, prepared_values = SqlQueryHelper.prepare_query(query_string, query_parameters) @@ -25,10 +25,10 @@ def prepare_count_query(query_string: str, query_parameters: Optional[Dict] = No @staticmethod def prepare_limit_query( query_string: str, - query_parameters: Optional[Dict] = None, + query_parameters: Optional[dict] = None, offset: Optional[int] = None, limit: Optional[int] = None, - ) -> Tuple[str, list]: + ) -> tuple[str, list]: """Build a new query by adding a select query with a limit above from input query""" prepared_query, prepared_values = SqlQueryHelper.prepare_query(query_string, query_parameters) query_check = prepared_query.strip().lower() @@ -43,7 +43,7 @@ def prepare_limit_query( return prepared_query, prepared_values @staticmethod - def prepare_query(query: str, query_parameters: Optional[Dict] = None) -> Tuple[str, list]: + def prepare_query(query: str, query_parameters: Optional[dict] = None) -> tuple[str, list]: """Prepare actual query by applying parameters and limit / offset restrictions""" query = convert_to_printf_templating_style(query) converted_query, ordered_values = convert_to_qmark_paramstyle(query, query_parameters) diff --git a/toucan_connectors/toucan_connector.py b/toucan_connectors/toucan_connector.py index eda02bf7e..00634aa9a 100644 --- a/toucan_connectors/toucan_connector.py +++ b/toucan_connectors/toucan_connector.py @@ -5,10 +5,11 @@ import socket import uuid from abc import ABC, ABCMeta, abstractmethod +from collections.abc import Iterable from enum import Enum from functools import reduce, wraps from types import ModuleType -from typing import TYPE_CHECKING, Annotated, Any, Generic, Iterable, NamedTuple, Type, TypeVar, Union +from typing import TYPE_CHECKING, Annotated, Any, Generic, NamedTuple, TypeVar, Union from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, SecretStr from pydantic.fields import ModelPrivateAttr @@ -298,7 +299,7 @@ def __init_subclass__(cls, /, *, data_source_model: type[DS]): name: str = Field(...) retry_policy: RetryPolicy | None = RetryPolicy() - _retry_on: Iterable[Type[BaseException]] = () + _retry_on: Iterable[type[BaseException]] = () type: str | None = None secrets_storage_version: str = Field("1", **UI_HIDDEN) diff --git a/toucan_connectors/utils/json_to_table.py b/toucan_connectors/utils/json_to_table.py index 6dea4e85b..3a1069c15 100644 --- a/toucan_connectors/utils/json_to_table.py +++ b/toucan_connectors/utils/json_to_table.py @@ -1,5 +1,5 @@ import uuid -from typing import TYPE_CHECKING, Any, List, Union +from typing import TYPE_CHECKING, Any, Union if TYPE_CHECKING: # pragma: no cover from pandas import DataFrame, Series @@ -13,7 +13,7 @@ def _first_valid_value(serie: "Series") -> Any: return serie[first_valid_index] if first_valid_index is not None else None -def json_to_table(df: "DataFrame", columns: Union[str, List[str]], sep: str = ".") -> "DataFrame": +def json_to_table(df: "DataFrame", columns: Union[str, list[str]], sep: str = ".") -> "DataFrame": """ Flatten JSON into a table shape. Add lines for each element of a nested array. Add columns for each keys of a nested object / dict. diff --git a/toucan_connectors/utils/pem.py b/toucan_connectors/utils/pem.py index 93e9c52c6..d084eb54f 100644 --- a/toucan_connectors/utils/pem.py +++ b/toucan_connectors/utils/pem.py @@ -1,4 +1,4 @@ -from typing import Iterable +from collections.abc import Iterable def _fix_pem(raw: str) -> Iterable[str]: