feat(ingest): loosen sqlalchemy dep & support airflow 2.3+ (datahub-p…

…roject#6204)

Co-authored-by: Tamas Nemeth <[email protected]>

.github/workflows/metadata-ingestion.yml

@@ -38,6 +38,11 @@ jobs:
             "testIntegrationBatch1",
             "testSlowIntegration",
           ]
+        include:
+          - python-version: "3.7"
+            extraPythonRequirement: "sqlalchemy==1.3.24 apache-airflow~=2.2.0"
+          - python-version: "3.10"
+            extraPythonRequirement: "sqlalchemy~=1.4.0 apache-airflow>=2.4.0"
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
@@ -50,8 +55,8 @@ jobs:
           hadoop-version: "3.2"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
-      - name: Run metadata-ingestion tests
-        run: ./gradlew :metadata-ingestion:build :metadata-ingestion:${{ matrix.command }}
+      - name: Run metadata-ingestion tests (extras ${{ matrix.extraPythonRequirement }})
+        run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion:${{ matrix.command }}
       - name: pip freeze show list installed
         if: always()
         run: source metadata-ingestion/venv/bin/activate && pip freeze

metadata-ingestion-modules/airflow-plugin/build.gradle

@@ -7,8 +7,10 @@ ext {
   venv_name = 'venv'
 }
 
+def pip_install_command = "${venv_name}/bin/pip install -e ../../metadata-ingestion"
+
 task checkPythonVersion(type: Exec) {
-  commandLine python_executable, '-c', 'import sys; assert sys.version_info >= (3, 6)'
+  commandLine python_executable, '-c', 'import sys; assert sys.version_info >= (3, 7)'
 }
 
 task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
@@ -20,7 +22,7 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
 task installPackage(type: Exec, dependsOn: environmentSetup) {
   inputs.file file('setup.py')
   outputs.dir("${venv_name}")
-  commandLine "${venv_name}/bin/pip", 'install', '-e', '.'
+  commandLine 'bash', '-x', '-c', "${pip_install_command} -e ."
 }
 
 task install(dependsOn: [installPackage])
@@ -30,7 +32,7 @@ task installDev(type: Exec, dependsOn: [install]) {
   outputs.dir("${venv_name}")
   outputs.file("${venv_name}/.build_install_dev_sentinel")
   commandLine 'bash', '-x', '-c',
-    "${venv_name}/bin/pip install -e .[dev] && touch ${venv_name}/.build_install_dev_sentinel"
+    "${pip_install_command} -e .[dev] && touch ${venv_name}/.build_install_dev_sentinel"
 }
 
 task lint(type: Exec, dependsOn: installDev) {
@@ -65,7 +67,7 @@ task installDevTest(type: Exec, dependsOn: [installDev]) {
   outputs.dir("${venv_name}")
   outputs.file("${venv_name}/.build_install_dev_test_sentinel")
   commandLine 'bash', '-x', '-c',
-    "${venv_name}/bin/pip install -e .[dev,integration-tests] && touch ${venv_name}/.build_install_dev_test_sentinel"
+    "${pip_install_command} -e .[dev,integration-tests] && touch ${venv_name}/.build_install_dev_test_sentinel"
 }
 
 def testFile = hasProperty('testFile') ? testFile : 'unknown'

metadata-ingestion-modules/airflow-plugin/setup.py

@@ -1,9 +1,9 @@
 import os
 import pathlib
-from typing import Dict, Set
 
 import setuptools
 
+
 package_metadata: dict = {}
 with open("./src/datahub_airflow_plugin/__init__.py") as fp:
     exec(fp.read(), package_metadata)
@@ -23,9 +23,7 @@ def get_long_description():
     "typing-inspect",
     "pydantic>=1.5.1",
     "apache-airflow >= 2.0.2",
-    "acryl-datahub[airflow] >= 0.8.36",
-    # Pinned dependencies to make dependency resolution faster.
-    "sqlalchemy==1.3.24",
+    f"acryl-datahub[airflow] == {package_metadata['__version__']}",
 }