chore: Refresh tap (#29)

* chore: Refresh tap

* Make Ruff happy

* Enable integration tests

* Update SQLAlchemy URL

* Constrain sqlalchemy

* Update default driver

* Install sqlalchemy mypy plugin

* Update triggers

.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: daily
+      time: "12:00"
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: weekly
+    labels: [deps]
+    groups:
+      actions:
+        patterns:
+          - "*"

.github/workflows/test.yml

@@ -1,12 +1,23 @@
 name: Test tap-myqsql
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened]
   push:
     branches: [main]
+    paths:
+    - tap_mysql/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - .github/workflows/test.yml
+  pull_request:
+    branches: [main]
+    paths:
+    - tap_mysql/**
+    - tests/**
+    - poetry.lock
+    - pyproject.toml
+    - .github/workflows/test.yml
   workflow_dispatch:
-    inputs: {}
 
 jobs:
   tests:
@@ -15,7 +26,7 @@ jobs:
       PIP_CONSTRAINT: .github/workflows/constraints.txt
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
     - name: Checkout code
@@ -24,15 +35,15 @@ jobs:
     - name: Fix Permissions
       run: |
         chmod 777 -R ./ssh_tunnel/ssh-server-config
-    
+
     - name: Set up MySQL containers (SSH Included)
       run: |
         docker compose -f docker-compose.yml up -d
 
     - uses: isbang/[email protected]
-  
+
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4.6.0
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -42,8 +53,12 @@ jobs:
         pip --version
     - name: Install Poetry
       run: |
-        pip install poetry
+        pipx install poetry
         poetry --version
+    - name: Install Tox
+      run: |
+        pipx install tox
+        tox --version
     - name: Install dependencies
       run: |
         poetry install
@@ -52,7 +67,6 @@ jobs:
         poetry run pytest
     - name: Run lint
       run: |
-        poetry run tox -e lint
-
+        tox -e typing
 
-# To test mysql client 1. docker exec -it 012c /bin/bash (ssh server) 2. apk add mysql mysql-client 3. mysql -h 10.5.0.5 -P 3306 -u root -ppassword 
+# To test mysql client 1. docker exec -it 012c /bin/bash (ssh server) 2. apk add mysql mysql-client 3. mysql -h 10.5.0.5 -P 3306 -u root -ppassword

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+ci:
+  autofix_prs: false
+  autoupdate_schedule: weekly
+  autoupdate_commit_msg: 'chore: pre-commit autoupdate'
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.5.0
+  hooks:
+  - id: check-json
+  - id: check-toml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.2.0
+  hooks:
+  - id: ruff
+    args: [--fix]
+  - id: ruff-format

docker-compose.yml

@@ -11,7 +11,7 @@ services:
       MYSQL_DATABASE: melty
       MYSQL_ROOT_HOST: '%'
     ports:
-      - 3306:3306 
+      - 3306:3306
   mysqldb_ssh:
     image: mysql
     restart: always

pyproject.toml

@@ -11,62 +11,67 @@ keywords = [
 license = "Apache-2.0"
 
 [tool.poetry.dependencies]
-python = "<3.12,>=3.8.1"
-singer-sdk = { version="^0.30.0" }
-mysqlclient = "^2.2.0"
-fs-s3fs = { version = "^1.1.1", optional = true }
+python = ">=3.8"
+fs-s3fs = { version = "==1.1.1", optional = true }
+singer-sdk = { version="~=0.34.0" }
+pymysql = "==1.1.0"
+sqlalchemy = "<2"
 sshtunnel = "0.4.0"
 
-[tool.poetry.dev-dependencies]
-pytest = "^7.3.2"
-tox = "^3.24.4"
-flake8 = "^5.0.4"
-flake8-docstrings = "^1.7.0"
-black = "23.1.0"
-mypy = "^1.0"
-isort = "^5.11.5"
-remote-pdb="2.1.0"
-
-[tool.isort]
-profile = "black"
-multi_line_output = 3 # Vertical Hanging Indent
-src_paths = "tap_mysql"
-
+# Binary client for MySQL
+mysqlclient = { version = "==2.2.0", optional = true }
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.2.1"
-faker = ">=18.5.1,<20.0.0"
-singer-sdk = { version="^0.30.0", extras = ["testing"] }
+faker = ">=20"
+pytest = ">=7.3.2"
+singer-sdk = { version="~=0.34.0", extras = ["testing"] }
+remote-pdb=">=2.1.0"
+
+[tool.poetry.group.typing.dependencies]
+mypy = ">=1.8.0"
+sqlalchemy = { version = "<2", extras = ["mypy"] }
+types-paramiko = ">=3.4.0.20240120"
 
 [tool.poetry.extras]
 s3 = ["fs-s3fs"]
+binary = ["mysqlclient"]
 
 [tool.mypy]
-python_version = "3.9"
+python_version = "3.12"
 warn_unused_configs = true
-plugins = "sqlmypy"
+plugins = "sqlalchemy.ext.mypy.plugin"  # TODO: Remove this when SQLAlchemy 2.0 is supported
 
 [tool.ruff]
+src = ["tap_mysql"]
+target-version = "py38"
+
+[[tool.mypy.overrides]]
+ignore_missing_imports = true
+module = [
+    "sshtunnel.*",  # https://github.com/pahaz/sshtunnel/issues/265
+]
+
+[tool.ruff.lint]
 ignore = [
     "ANN101",  # missing-type-self
     "ANN102",  # missing-type-cls
+    # Conflict with Ruff's formatter
+    "COM812",
+    "ISC001",
 ]
 select = ["ALL"]
-src = ["tap_mysql"]
-target-version = "py37"
-
 
-[tool.ruff.flake8-annotations]
+[tool.ruff.lint.flake8-annotations]
 allow-star-arg-any = true
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 known-first-party = ["tap_mysql"]
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "google"
 
 [build-system]
-requires = ["poetry-core>=1.0.8"]
+requires = ["poetry-core==1.9.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.scripts]

tap_mysql/client.py

@@ -11,7 +11,6 @@
 from singer_sdk.helpers._typing import TypeConformanceLevel
 
 if TYPE_CHECKING:
-    from sqlalchemy.dialects import mysql
     from sqlalchemy.engine import Engine
     from sqlalchemy.engine.reflection import Inspector
 
@@ -47,12 +46,11 @@ def to_jsonschema_type(
         sql_type: str  # noqa: ANN401
         | sqlalchemy.types.TypeEngine
         | type[sqlalchemy.types.TypeEngine]
-        | mysql.ARRAY
         | Any,
     ) -> dict:
         """Return a JSON Schema representation of the provided type.
 
-        Overidden from SQLConnector to correctly handle JSONB and Arrays.
+        Overridden from SQLConnector to correctly handle JSONB and Arrays.
 
         By default will call `typing.to_jsonschema_type()` for strings and SQLAlchemy
         types.

tap_mysql/tap.py

@@ -172,7 +172,7 @@ def get_sqlalchemy_url(self, config: Mapping[str, Any]) -> str:
             return cast(str, config["sqlalchemy_url"])
 
         sqlalchemy_url = URL.create(
-            drivername="mysql+mysqldb",
+            drivername="mysql+pymysql",
             username=config["user"],
             password=config["password"],
             host=config["host"],
@@ -222,7 +222,7 @@ def guess_key_type(self, key_data: str) -> paramiko.PKey:
             paramiko.Ed25519Key,
         ):
             try:
-                key = key_class.from_private_key(io.StringIO(key_data))  # type: ignore[attr-defined]  # noqa: E501
+                key = key_class.from_private_key(io.StringIO(key_data))  # type: ignore[attr-defined]
             except paramiko.SSHException:  # noqa: PERF203
                 continue
             else:

tests/test_core.py

@@ -22,7 +22,7 @@
     "start_date": pendulum.datetime(2022, 11, 1).to_iso8601_string(),
     # Using 127.0.0.1 instead of localhost because of mysqlclient dialect.
     # See: https://stackoverflow.com/questions/72294279/how-to-connect-to-mysql-databas-using-github-actions
-    "sqlalchemy_url": "mysql+mysqldb://root:[email protected]:3306/melty",
+    "sqlalchemy_url": "mysql+pymysql://root:[email protected]:3306/melty",
 }
 
 NO_SQLALCHEMY_CONFIG = {

tests/test_replication_key.py

@@ -13,7 +13,7 @@
     "start_date": pendulum.datetime(2022, 11, 1).to_iso8601_string(),
     # Using 127.0.0.1 instead of localhost because of mysqlclient dialect.
     # See: https://stackoverflow.com/questions/72294279/how-to-connect-to-mysql-databas-using-github-actions
-    "sqlalchemy_url": f"mysql+mysqldb://root:[email protected]:3306/melty",
+    "sqlalchemy_url": f"mysql+pymysql://root:[email protected]:3306/melty",
 }
 
 

tests/test_selected_columns_only.py

@@ -14,7 +14,7 @@
 SAMPLE_CONFIG = {
     # Using 127.0.0.1 instead of localhost because of mysqlclient dialect.
     # See: https://stackoverflow.com/questions/72294279/how-to-connect-to-mysql-databas-using-github-actions
-    "sqlalchemy_url": f"mysql+mysqldb://root:[email protected]:3306/melty",
+    "sqlalchemy_url": f"mysql+pymysql://root:[email protected]:3306/melty",
 }
 
 

tests/test_ssh_tunnel.py

@@ -5,7 +5,7 @@
 
 TABLE_NAME = "test_replication_key"
 SAMPLE_CONFIG = {
-    "sqlalchemy_url": "mysql+mysqldb://root:[email protected]:3306/melty",
+    "sqlalchemy_url": "mysql+pymysql://root:[email protected]:3306/melty",
     "ssh_tunnel": {
         "enable": True,
         "host": "127.0.0.1",

tox.ini

@@ -1,48 +1,15 @@
 # This file can be used to customize tox tests as well as other test frameworks like flake8 and mypy
 
 [tox]
-envlist = py39
-; envlist = py37, py38, py39
-isolated_build = true
+envlist = py38,py39,py310,py311,py312
 
 [testenv]
-whitelist_externals = poetry
-
-commands =
-    poetry install -v
-    poetry run pytest
-    poetry run black --check tap_mysql
-    poetry run flake8  tap_mysql
-
-[testenv:pytest]
-# Run the python tests.
-# To execute, run `tox -e pytest`
-envlist = py37, py38, py39
+allowlist_externals = poetry
 commands =
     poetry install -v
     poetry run pytest
 
-[testenv:format]
-# Attempt to auto-resolve lint errors before they are raised.
-# To execute, run `tox -e format`
+[testenv:typing]
 commands =
-    poetry install -v
-    poetry run black tap_mysql
-    poetry run isort tap_mysql
-
-[testenv:lint]
-# Raise an error if lint and style standards are not met.
-# To execute, run `tox -e lint`
-commands =
-    poetry install -v
-    poetry run black --check --diff tap_mysql/
-    poetry run isort --check tap_mysql
-    poetry run flake8 tap_mysql
-
-[flake8]
-ignore = W503
-max-line-length = 88
-max-complexity = 10
-
-[pydocstyle]
-ignore = D105,D203,D213
+    poetry install -v --with typing
+    poetry run mypy tap_mysql